# -*- coding: utf8 -*-
import urllib2
import urllib
import re
import os
import time
import download
import download2
from Queue import Queue
import CatchXmlwbk
def down(url):
    queue1=Queue()
    content=urllib2.urlopen(url).read()
    name_pat=re.compile(r'title>(.*?)</tit')
    category=re.findall(name_pat,content)[0]
    site_pat=re.compile(r' href="(.*?)" target="_blank">')
    sites=re.findall(site_pat,content)
    path = "F:\Python27\Zgr03"
    title =category+'T'
    category=title.decode('utf-8')
    new_path = os.path.join(path, category)
    if not os.path.isdir(new_path):
        os.makedirs(new_path)
    filename=path+os.sep+category+os.sep+'1.txt'
    '''
    pointfile=open(filename,'w')#初始化断点文件
    pointfile.write('0')
    pointfile.close()
    '''
    file = open(filename,'r')
    Strpoint=file.readline()
    file.close()
    Breakpoint=int(Strpoint)
    if Breakpoint==len(sites):
        return 0
    for x in sites:
        queue1.put(x)
    #print('name size')
    #print(queue1.qsize())
    for i in range(Breakpoint):
        queue1.get()
    #print(queue1.qsize())
    while(queue1.qsize()>0):
        #print(queue1.qsize())
        x=queue1.get()
        try:
            download.download(x,category)
        except:
            try:
                download2.download(x,category)
                print('download2')
            except:
                time.sleep(0.001)
                print('sleeping'+category+x)
                queue1.put(x)
        pointfile=open(filename,'w')
        pointfile.write(str(len(sites)-queue1.qsize()))
        pointfile.close
    #print('finish')
    return 1
url='http://fenlei.baike.com/'
sites=CatchXmlwbk.add_func(url)
print(sites)
for x in sites:
    down(x)
exit()
